{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial\n",
    "MILA, November 2017\n",
    "\n",
    "By Sandeep Subramanian"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Introduction to the torch tensor library\n",
    "### Torch's numpy equivalent with GPU support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize a random tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 2.4878e+04  4.5692e-41  2.4878e+04\n",
       " 4.5692e-41 -2.9205e+19  4.5691e-41\n",
       " 1.2277e-02  4.5692e-41 -4.0170e+19\n",
       " 4.5691e-41  1.2277e-02  4.5692e-41\n",
       " 0.0000e+00  0.0000e+00  0.0000e+00\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(5, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From a uniform distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "-0.2767 -0.1082 -0.1339\n",
       "-0.6477  0.3098  0.1642\n",
       "-0.1125 -0.2104  0.8962\n",
       "-0.6573  0.9669 -0.3806\n",
       " 0.8008 -0.3860  0.6816\n",
       "[torch.FloatTensor of size 5x3]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(5, 3).uniform_(-1, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get it's shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.Tensor(5, 3).uniform_(-1, 1)\n",
    "print(x.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor Types\n",
    "source: http://pytorch.org/docs/master/tensors.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Data type |Tensor|\n",
    "|----------|------|\n",
    "|32-bit floating point|\ttorch.FloatTensor|\n",
    "|64-bit floating point|\ttorch.DoubleTensor|\n",
    "|16-bit floating point|\ttorch.HalfTensor|\n",
    "|8-bit integer (unsigned)|torch.ByteTensor|\n",
    "|8-bit integer (signed)|torch.CharTensor|\n",
    "|16-bit integer (signed)|torch.ShortTensor|\n",
    "|32-bit integer (signed)|torch.IntTensor|\n",
    "|64-bit integer (signed)|torch.LongTensor|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creation from lists & numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.LongTensor\n",
      "int64\n"
     ]
    }
   ],
   "source": [
    "z = torch.LongTensor([[1, 3], [2, 9]])\n",
    "print(z.type())\n",
    "# Cast to numpy ndarray\n",
    "print(z.numpy().dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.DoubleTensor\n",
      "torch.FloatTensor\n"
     ]
    }
   ],
   "source": [
    "# Data type inferred from numpy\n",
    "print(torch.from_numpy(np.random.rand(5, 3)).type())\n",
    "print(torch.from_numpy(np.random.rand(5, 3).astype(np.float32)).type())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple mathematical operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.2200 -0.0368  0.4494\n",
      "-0.2577 -0.0343  0.1587\n",
      "-0.7503 -0.1729  0.0453\n",
      " 0.9296 -0.1067 -0.6402\n",
      "-0.3276  0.0158 -0.0552\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y = x * torch.randn(5, 3)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.2820 -0.1633 -4.4346\n",
      "-1.6809  0.2066 -0.8261\n",
      "-0.6464  0.9758  0.2542\n",
      " 0.5789  0.1890 -0.4662\n",
      " 5.3183  0.0236 -0.1403\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y = x / torch.sqrt(torch.randn(5, 3) ** 2)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Broadcasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 3])\n",
      "\n",
      " 0.1919 -0.5006 -1.2410\n",
      "-0.8080  0.1407 -0.6193\n",
      "-1.6629 -0.1580 -0.3921\n",
      " 1.0395  0.7069 -0.1459\n",
      " 1.9027  1.4343  1.2299\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print (x.size())\n",
    "y = x + torch.randn(5, 1)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 10, 15])\n",
      "torch.Size([50, 15])\n",
      "torch.Size([50, 1, 15])\n",
      "torch.Size([50, 15])\n",
      "\n",
      "torch.Size([10, 5, 15])\n",
      "torch.Size([5, 15, 10])\n",
      "torch.Size([10, 15, 5])\n",
      "torch.Size([10, 15, 5])\n"
     ]
    }
   ],
   "source": [
    "y = torch.randn(5, 10, 15)\n",
    "print(y.size())\n",
    "print(y.view(-1, 15).size())  # Same as doing y.view(50, 15)\n",
    "print(y.view(-1, 15).unsqueeze(1).size()) # Adds a dimension at index 1.\n",
    "print(y.view(-1, 15).unsqueeze(1).squeeze().size())\n",
    "# If input is of shape: (Ax1xBxCx1xD)(Ax1xBxCx1xD) then the out Tensor will be of shape: (AxBxCxD)(AxBxCxD)\n",
    "print()\n",
    "print(y.transpose(0, 1).size())\n",
    "print(y.transpose(1, 2).size())\n",
    "print(y.transpose(0, 1).transpose(1, 2).size())\n",
    "print(y.permute(1, 2, 0).size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([50, 100, 15])\n",
      "torch.Size([50, 100, 15])\n"
     ]
    }
   ],
   "source": [
    "print(y.view(-1, 15).unsqueeze(1).expand(50, 100, 15).size())\n",
    "print(y.view(-1, 15).unsqueeze(1).expand_as(torch.randn(50, 100, 15)).size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Concatenate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 10, 30])\n",
      "torch.Size([2, 5, 10, 15])\n"
     ]
    }
   ],
   "source": [
    "# 2 is the dimension over which the tensors are concatenated\n",
    "print(torch.cat([y, y], 2).size())\n",
    "# stack concatenates the sequence of tensors along a new dimension.\n",
    "print(torch.stack([y, y], 0).size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Advanced Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 4])\n",
      "torch.Size([2, 3, 4])\n"
     ]
    }
   ],
   "source": [
    "y = torch.randn(2, 3, 4)\n",
    "print(y[[1, 0, 1, 1]].size())\n",
    "\n",
    "# PyTorch doesn't support negative strides yet so ::-1 does not work.\n",
    "rev_idx = torch.arange(1, -1, -1).long()\n",
    "print(y[rev_idx].size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPU support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.2456  1.1543  0.5376  0.4358 -0.0369\n",
       " 0.8247 -0.4143 -0.7188  0.3953  0.2573\n",
       "-0.1346  0.7329  0.5156  0.0864 -0.1349\n",
       "-0.3555  0.3135  0.3921 -0.1428 -0.1368\n",
       "-0.4385  0.5601  0.6533 -0.2793 -0.5220\n",
       "[torch.cuda.HalfTensor of size 5x5 (GPU 0)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.cuda.HalfTensor(5, 3).uniform_(-1, 1)\n",
    "y = torch.cuda.HalfTensor(3, 5).uniform_(-1, 1)\n",
    "torch.matmul(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Move tensors on the CPU -> GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-0.3758 -0.1090  0.7911\n",
      " 0.2839 -0.9136  0.1070\n",
      " 0.9184  0.5113 -0.8040\n",
      "-0.3412 -0.8895 -0.5780\n",
      "-0.0992  0.0983  0.6074\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n",
      "\n",
      "-0.3758 -0.1090  0.7911\n",
      " 0.2839 -0.9136  0.1070\n",
      " 0.9184  0.5113 -0.8040\n",
      "-0.3412 -0.8895 -0.5780\n",
      "-0.0992  0.0983  0.6074\n",
      "[torch.cuda.FloatTensor of size 5x3 (GPU 0)]\n",
      "\n",
      "\n",
      "-0.3758 -0.1090  0.7911\n",
      " 0.2839 -0.9136  0.1070\n",
      " 0.9184  0.5113 -0.8040\n",
      "-0.3412 -0.8895 -0.5780\n",
      "-0.0992  0.0983  0.6074\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x = torch.FloatTensor(5, 3).uniform_(-1, 1)\n",
    "print(x)\n",
    "x = x.cuda(device=0)\n",
    "print(x)\n",
    "x = x.cpu()\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Contiguity in memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.4740 -0.9209  0.4143\n",
      "-0.3473  0.4474 -0.8159\n",
      "-0.7654 -0.0956  0.6145\n",
      "-0.0846 -0.6239  0.8609\n",
      "-0.8142  0.9289 -0.7020\n",
      "[torch.FloatTensor of size 5x3]\n",
      "\n",
      "\n",
      " 0.4740 -0.9209  0.4143\n",
      "-0.3473  0.4474 -0.8159\n",
      "-0.7654 -0.0956  0.6145\n",
      "-0.0846 -0.6239  0.8609\n",
      "-0.8142  0.9289 -0.7020\n",
      "[torch.cuda.FloatTensor of size 5x3 (GPU 0)]\n",
      "\n",
      "Contiguity : True \n",
      "Contiguity : False \n",
      "Contiguity : True \n"
     ]
    }
   ],
   "source": [
    "x = torch.FloatTensor(5, 3).uniform_(-1, 1)\n",
    "print(x)\n",
    "x = x.cuda(device=0)\n",
    "print(x)\n",
    "print('Contiguity : %s ' % (x.is_contiguous()))\n",
    "x = x.unsqueeze(0).expand(30, 5, 3)\n",
    "print('Contiguity : %s ' % (x.is_contiguous()))\n",
    "x = x.contiguous()\n",
    "print('Contiguity : %s ' % (x.is_contiguous()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
